package ca.uwindsor.cs.deepweb.utility;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.Date;
import java.util.zip.GZIPInputStream;

public class FileToDcoumentExtractorforGOV2 {

	private String destinationDirectory;

	private String newDirectory;

	protected BufferedReader bufferedreader_in;

	public FileToDcoumentExtractorforGOV2(String dir) {
		destinationDirectory = dir;
		newDirectory = new String();
	}

	public void start(String sourcedir, String destdir) {
		File[] list = new File(sourcedir).listFiles();
		for (int index = 0; index < list.length; index++) {
			if (list[index].isDirectory()) {
				newDirectory = destdir
//						+ File.separator
						+ list[index].getAbsolutePath().substring(list[index].getAbsolutePath().lastIndexOf(File.separator), list[index].getAbsolutePath().length());
				File dir = new File(newDirectory);
				dir.mkdir();

				start(list[index].getAbsolutePath(),newDirectory);
			} else if (list[index].isFile()) {
				if (list[index].getName().endsWith(".gz")) {
					go(list[index].getAbsolutePath() ,destdir);
					
				} else {
					System.out.println(list[index].getAbsolutePath());
				}
			} else {
				System.out.println(list[index].getAbsolutePath());
			}
		}
	}

	// public void start(String dir) {
	// File[] list = new File(dir).listFiles();
	// for (int index = 0; index < list.length; index++) {
	// if (list[index].isDirectory()) {
	// start(list[index].getAbsolutePath());
	// } else if (list[index].isFile()) {
	// if (list[index].getName().endsWith(".gz")) {
	// go(list[index].getAbsolutePath());
	// } else {
	// System.out.println(list[index].getAbsolutePath());
	// }
	// } else {
	// System.out.println(list[index].getAbsolutePath());
	// }
	// }
	// }

	private void go(String gzfilename, String directory) {
		// create directory
		newDirectory = directory
				+ File.separator
				+ gzfilename.substring(gzfilename.lastIndexOf(File.separator),
						gzfilename.lastIndexOf(".gz"));
		File dir = new File(newDirectory);
		dir.mkdir();

		uncompressFile(gzfilename);
		readuncompressedfile();

	}

	private void uncompressFile(String gzfilename) {
		try {
			// Open the compressed file
			// String inFilename = "infile.gzip";
			GZIPInputStream in = new GZIPInputStream(new FileInputStream(
					gzfilename));

			// Open the output file
			String outFilename = "d:\\temp";
			OutputStream out = new FileOutputStream(outFilename);

			// Transfer bytes from the compressed file to the output file
			byte[] buf = new byte[1024];
			int len;
			while ((len = in.read(buf)) > 0) {
				out.write(buf, 0, len);
			}

			// Close the file and stream
			in.close();
			out.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void readuncompressedfile() {
		InputStreamReader reader = null;
		String string_line = new String();
		try {
			reader = new InputStreamReader(new FileInputStream("d:\\temp"));
			bufferedreader_in = new BufferedReader(reader);
			string_line = bufferedreader_in.readLine();
			StringBuffer strbuffer = new StringBuffer();

			while (string_line != null) {
				if (string_line.contains("</DOC>")) {
					generateDocument(strbuffer.toString());
					strbuffer = new StringBuffer();
				} else {
					strbuffer.append(string_line);
					strbuffer.append("\n");
				}
				string_line = bufferedreader_in.readLine();
			}

		} catch (IOException ex) {
			ex.printStackTrace();
		}
	}

	private void generateDocument(String string) {

		String[] lines = string.split("\n");
		String fileid = lines[1].substring(7, lines[1].length() - 8);
		System.out.println(fileid);
		StringBuffer docbuffer = new StringBuffer();
		// boolean isfound;
		int indexj;
		for (indexj = 0; indexj < lines.length; indexj++) {
			if (lines[indexj].contains("</DOCHDR>")) {
				indexj++;
				break;
			}
		}
		for (; indexj < lines.length; indexj++) {
			docbuffer.append(lines[indexj]);
			docbuffer.append("\n");
		}
		// docbuffer.append("</html>");
		// if(fileid.equals("GX000-00-15597212")){
		// System.out.println(lines);
		// }
		StringBuffer filename = new StringBuffer();
		filename.append(newDirectory);
		filename.append(java.io.File.separator);
		filename.append(fileid);
		filename.append(".html");
		try {
			FileOutputStream fp = new FileOutputStream(filename.toString());
			fp.write(docbuffer.toString().getBytes("UTF-8"));
			fp.flush();
			fp.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		// String inFilename = "D:\\ѧϰ����\\Graduate
		// Study\\Research\\Data\\GOV2\\00.gz";
		// String outDir = "D:\\ѧϰ����\\Graduate Study\\Research\\Data\\test";
		args = new String[2];
		args[0] = "Z:\\DOTGOV2\\gov2-corpus";
		args[1] = "F:\\gov2-corpus-extracted";
		long startTime = new Date().getTime();
		String startDIR = args[0];
		String outDir = args[1];
		FileToDcoumentExtractorforGOV2 extractor = new FileToDcoumentExtractorforGOV2(
				outDir);
		extractor.start(startDIR, outDir);
		long endTime = new Date().getTime();

		System.out.println("It takes " + (endTime - startTime)
				+ " milliseconds to extrat the files from" + startDIR);

	}

}
